/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

package utfpr.cp.crawler;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import java.util.logging.Level;
import java.util.logging.Logger;
import utfpr.cp.control.TargetControl;
import utfpr.cp.model.URLTarget;

/**
 *
 * @author Adriano_2
 */
public class CrawlerMonitor {
    
    protected static CrawlController createController(URLTarget page) throws Exception {
        CrawlConfig config = new CrawlConfig();
        Properties prop = Util.getCrawlerConfig();
        
        // Set properties
        config.setCrawlStorageFolder(prop.getProperty("crawlStorageFolder") + "/" + Util.cleanURL(page.getUrl()));
        int delay = Integer.parseInt(prop.getProperty("politenessDelay"));
        if (delay > 0) {
            config.setPolitenessDelay(delay);
        }
        int limit = Integer.parseInt(prop.getProperty("pagesSiteLimit"));
        if (limit > 0) {
            config.setMaxPagesToFetch(limit);
        }
        
        //Set proxy config
        if (prop.containsKey("proxyConfig")) {
            Properties proxyConfig = Util.getProperties(prop.getProperty("proxyConfig"));
            if (!proxyConfig.isEmpty()) {
                config.setProxyHost(proxyConfig.getProperty("host"));
                config.setProxyPort(Integer.parseInt(proxyConfig.getProperty("port", "3128")));
                if (proxyConfig.containsKey("username")) {
                    config.setProxyUsername(proxyConfig.getProperty("username"));
                }
                if (proxyConfig.containsKey("password")) {
                    config.setProxyPassword(proxyConfig.getProperty("password"));
                }
            }
        }
        
        //Create Robots
        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtServer robotstxtServer = new RobotstxtServer(new RobotstxtConfig(), pageFetcher);
        
        //Create controller
        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
        controller.addSeed(page.getUrl());
        
        //Set custom data
        ThreadManager ctl = new SingleSiteControl(page, prop.getProperty("ignoreExtension"));
        controller.setCustomData(ctl);
         
        return controller;
    }
    
    final CrawlerStats oldStats;
    final HashMap<String, CrawlController> controls;
    private boolean status;
    private int domains;
    private int threads;
    
    public CrawlerMonitor() {
        controls = new HashMap<>();
        status = false;
        oldStats = new CrawlerStats(0);
    }
    
    public boolean start(int domains, int threads) {
        Util.showDebug("Start Monitor");
        if (isActive()) {
            return false;
        }
        this.domains = domains;
        this.threads = threads;
        for (int i = 0; i < domains; i++) {
            if (!createNewControl()) {
                i--;
            }
        }
        status = true;
        return true;
    }
    
    public void stop() {
        Iterator<String> iterator = controls.keySet().iterator();
        while (iterator.hasNext()) {
            String site = iterator.next();
            try {
                stopCrawler(controls.get(site));
            } catch (ConcurrentModificationException err) {
                iterator.remove();
            }
        }
        status = false;
    }
    
    public CrawlerStats controlCrawlerStatus() {
        CrawlerStats resumo = new CrawlerStats(0);
        resumo.merge(oldStats);
        for (String site : controls.keySet()) {
            CrawlController ctl = controls.get(site);
            resumo.merge(((ThreadManager)ctl.getCustomData()).getStats());
            if (ctl.isFinished()) {
                stopCrawler(ctl);
                createNewControl();
            }
            //TODO test other cases
        }
        return resumo;
    }
    
    public HashMap<String, CrawlerStats> getCrawlerStatusByDomain() {
        HashMap<String, CrawlerStats> result = new HashMap<>();
        for (String site : controls.keySet()) {
             CrawlController ctl = controls.get(site);
             CrawlerStats stats = ((ThreadManager)ctl.getCustomData()).getStats();
             result.put(stats.getDomain(), stats);
        }
        return result;
    }
    
    protected void startCrawler(CrawlController control, int threads) {
        Util.showDebug("Starting: "+((ThreadManager)control.getCustomData()).getTarget().getUrl());
        control.startNonBlocking(PageDownloader.class, threads);
    }
    
    protected void stopCrawler(CrawlController control) {
        SingleSiteControl ctl = (SingleSiteControl)control.getCustomData();
        CrawlerStats stats = ctl.getStats();
        Util.showDebug("Stopping: "+stats.getDomain());
        controls.remove(stats.getDomain());
        oldStats.merge(stats);
        
        if (control.isFinished()) {
           TargetControl.setFinished(ctl.getTarget());
           if (stats.getTotalSavedPages() < 100 || stats.getTotalAllErrors() > stats.getTotalSavedPages()) {
               TargetControl.setError(ctl.getTarget());
               return;
           }
        }
        else {
            control.shutdown();
            control.waitUntilFinish();
        }
        
        if (stats.getTotalSavedPages() > 2000) {
            TargetControl.setFinished(ctl.getTarget());
        }
        else {
            TargetControl.cancelStarted(ctl.getTarget());
        }
    }
    
    /**
     * @return the status
     */
    public boolean isActive() {
        return status;
    }

    private boolean createNewControl() {
        URLTarget site = TargetControl.getNextDomain();
        try {
            CrawlController ctl = createController(site);
            if (controls.containsKey(site.getUrl())) {
                //TODO
            }
            controls.put(site.getUrl(), ctl);
            startCrawler(ctl, threads);
            return true;
        } catch (Exception ex) {
            Logger.getLogger(CrawlerMonitor.class.getName()).log(Level.SEVERE, null, ex);
            return false;
        }
    }
}
